﻿Imports Aricie.DNN.Modules.LuceneSearch.Business
Imports LuceneNet = Lucene.Net
Imports LuceneSearch.Monitoring.Monitors

Namespace Components.Lucene.IndexMerger

    Public Class IndexMerger
        Implements IDisposable

        Private Property CurrentIndexWriter As LuceneNet.Index.IndexWriter
        Private Property Monitor As IndexMergerMonitor
        Private Property CurrentPortalIndexingDriver As PortalIndexingDriver
        Private Property MergeStartTime As DateTime = DateTime.Now

        Public Sub New(PID As PortalIndexingDriver)
            CurrentPortalIndexingDriver = PID
            Monitor = New IndexMergerMonitor(PID.LucenePortalSettings)
            CurrentIndexWriter = New LuceneNet.Index.IndexWriter(PID.GetDirectory, PID.GetPortalAnalyzer(), False, LuceneNet.Index.IndexWriter.MaxFieldLength.UNLIMITED)
        End Sub

        ''' <summary>
        ''' Merges current indexable items with existing index in order to take into account items that may have changed or been rendered obsolete
        ''' </summary>
        ''' <param name="itemsToIndex"></param>
        ''' <param name="CurrentIndexStructure"></param>
        ''' <remarks></remarks>
        Public Sub MergeNewItemsWithExistingIndexStructure(itemsToIndex As Global.DotNetNuke.Services.Search.SearchItemInfoCollection, CurrentIndexStructure As ExistingIndex.ExistingIndexStructure(Of ExistingIndex.ExistingIndexDocument))

            ' Deleting duplicate entries
            For Each UniqueIdentifierDuplicated In CurrentIndexStructure.DuplicateEntries
                Try
                    DeleteDocument(UniqueIdentifierDuplicated)
                    Monitor.DeletedForDuplication += 1 'It is impossible to know whether this number is correct. Assume it is for now
                Catch ex As Exception
                    Global.DotNetNuke.Services.Exceptions.Exceptions.LogException(New ApplicationException(String.Format("Error while deleting DUPLICATE docs with Uid {0}", UniqueIdentifierDuplicated), ex))
                End Try
            Next
            CurrentIndexWriter.Commit()

            'HACK: in order to get rid of processed LuceneSearchItemsInfo as quickly as possible, we create a single empty ItemInfo that
            ' will be assigned to the working LuceneSearchItemInfo after it's been dealt with
            Dim LuceneSearchItemInfoProcessed = New LuceneSearchItemInfo(CurrentPortalIndexingDriver.PortalId)

            Dim itemsProcessedIds = New HashSet(Of String) ' we're going to keep the ids of processed documents in order to delete documents that are no longer returned by the first phase of the search
            ' let's add what has been processed to our index
            For itemCounter As Integer = 0 To itemsToIndex.Count - 1
                Dim SII As LuceneSearchItemInfo = DirectCast(itemsToIndex(itemCounter), LuceneSearchItemInfo)

                ' if a document with the same id has already been processed we skip indexation to avoid duplicates
                If Not itemsProcessedIds.Contains(SII.UniqueId) Then
                    'If the document status is ok and if we still have time to add documents to our index
                    If SII.IndexingStatus = IndexingItemStatus.Ok AndAlso Not Monitor.StorageTimeExceeded Then
                        ' first we check whether or not previous documents exist in the index and may collide with our data
                        If (CurrentIndexStructure.UidToDocument.ContainsKey(SII.UniqueId)) Then
                            ' there is an older document matching the current one, let's check its hashcode and age in order to delete it if applicable
                            MergeDocumentWithPreviousIndexDocument(SII.ToDocument(), CurrentIndexStructure.UidToDocument(SII.UniqueId))
                        Else
                            ' no previous document match our signature, we may proceed to add the document to the index
                            AddDocument(SII.ToDocument())
                        End If
                    Else
                        Monitor.SkippedDocuments += 1
                    End If
                    Monitor.ProcessedDocuments += 1
                    itemsProcessedIds.Add(SII.UniqueId)
                End If

                ' if we don't need this item anymore, we could clear the item eagerly to free up memory
                ' WARNING: this will disable reflexive items providers, ie providers that are based upon existing items
                'TODO: disable this
                itemsToIndex(itemCounter) = LuceneSearchItemInfoProcessed
            Next
            CurrentIndexWriter.Commit()

            ' any document previously in the index that we didn't process is an obsolete document, we will delete it
            For Each DocumentToDeleteUniqueId In CurrentIndexStructure.UidToDocument.Keys.Except(itemsProcessedIds)
                Try
                    DeleteDocument(DocumentToDeleteUniqueId)
                    Monitor.DeletedForDisappearing += 1
                Catch ex As Exception
                    Global.DotNetNuke.Services.Exceptions.Exceptions.LogException(New ApplicationException(String.Format("Error while deleting REMOVED docs with Uid {0}", DocumentToDeleteUniqueId), ex))
                End Try

            Next
            CurrentIndexWriter.Commit()

            ' log some information regarding storage
            LogMonitorInformation()
        End Sub

        Private Sub MergeDocumentWithPreviousIndexDocument(latestDocument As LuceneNet.Documents.Document, previousDocument As ExistingIndex.ExistingIndexDocument)
            ' if hashcode changed, the content has changed
            If previousDocument.Hashcode <> latestDocument.Get(LuceneGeneralField.HashCode) Then
                ' we check however whether the publication date are different
                If previousDocument.PublicationDate <> LuceneNet.Documents.DateTools.StringToDate(latestDocument.Get(DnnSearchItemField.PubDate)) Then
                    Dim PublicationHistory = previousDocument.PublicationHistory
                    If Not String.IsNullOrEmpty(PublicationHistory) Then
                        PublicationHistory &= ","
                    End If
                    PublicationHistory &= LuceneNet.Documents.DateTools.DateToString(previousDocument.PublicationDate, LuceneNet.Documents.DateTools.Resolution.SECOND)
                    latestDocument.Add(FieldFactory.Instance.CreateField(LuceneGeneralField.PubHistory, PublicationHistory))
                End If
                Try
                    UpdateDocument(previousDocument.Uid, latestDocument)
                    Monitor.DeletedForObsolescence += 1
                Catch ex As Exception
                    Global.DotNetNuke.Services.Exceptions.Exceptions.LogException(New ApplicationException(String.Format("Error while updating too old document: Uid {0}", previousDocument.Uid), ex))
                End Try
            Else
                ' ok, hashcodes are the same, we will now check whether the previous item is too old or if we _need_ to delete more elements to force the index to refresh
                If (MergeStartTime - previousDocument.IndexedDate) > CurrentPortalIndexingDriver.LucenePortalSettings.MaxIndexedAge OrElse Monitor.DeletedDocuments < CurrentPortalIndexingDriver.LucenePortalSettings.MinDocTurnOver Then
                    'Existing document has not changed yet it is too old. Index again (which should only update the Indexed Date)
                    If String.IsNullOrEmpty(latestDocument.Get(LuceneGeneralField.ModifiedDate)) Then
                        ' if the latest document doesn't have a modification date, we pick the modification date from the previous document
                        Dim PreviousModificationDate = previousDocument.ModifiedDate
                        If PreviousModificationDate = DateTime.MinValue Then
                            PreviousModificationDate = DateTime.Now
                        End If
                        latestDocument.Add(FieldFactory.Instance.CreateField(LuceneGeneralField.ModifiedDate, LuceneNet.Documents.DateTools.DateToString(PreviousModificationDate, LuceneNet.Documents.DateTools.Resolution.SECOND)))
                    End If

                    Try
                        UpdateDocument(previousDocument.Uid, latestDocument)
                        Monitor.DeletedForAge += 1
                    Catch ex As Exception
                        Global.DotNetNuke.Services.Exceptions.Exceptions.LogException(New ApplicationException(String.Format("Error while updating obsolete document: Uid {0}", previousDocument.Uid), ex))
                    End Try
                Else
                    'There is no reason to change the existing document, hashcode hasn't been changed and its age is still all right
                    ' so we do nothing
                    Monitor.IgnoredDocuments += 1
                End If
            End If
        End Sub

#Region "Add, delete and update documents in Lucene index"
        Private Sub UpdateDocument(UniqueIdentifier As String, CurrentDocument As LuceneNet.Documents.Document)
            Monitor.AddWatch.Start()

            AddDateInformationToCurrentDocument(CurrentDocument)
            CurrentIndexWriter.UpdateDocument(New LuceneNet.Index.Term(Aricie.DNN.Modules.LuceneSearch.Business.LuceneGeneralField.Uid, UniqueIdentifier), CurrentDocument, CurrentPortalIndexingDriver.GetDocAnalyzer(CurrentDocument))

            Monitor.AddWatch.Stop()
            Monitor.UpdatedDocuments += 1

            CheckWhetherCommitIsNeeded()
            CheckStorageTimeAvailable()
        End Sub

        Private Sub DeleteDocument(UniqueIdentifier As String)
            Monitor.DeleteWatch.Start()
            CurrentIndexWriter.DeleteDocuments(New LuceneNet.Index.Term(Aricie.DNN.Modules.LuceneSearch.Business.LuceneGeneralField.Uid, UniqueIdentifier))
            Monitor.DeleteWatch.Stop()
        End Sub

        Private Sub AddDocument(CurrentDocument As LuceneNet.Documents.Document)
            Monitor.AddWatch.Start() ' measuring the storage time

            AddDateInformationToCurrentDocument(CurrentDocument)
            CurrentIndexWriter.AddDocument(CurrentDocument, CurrentPortalIndexingDriver.GetDocAnalyzer(CurrentDocument))

            Monitor.AddWatch.Stop() ' storage is done
            Monitor.AddedDocuments += 1

            CheckWhetherCommitIsNeeded()
            CheckStorageTimeAvailable()
        End Sub

        Private Sub AddDateInformationToCurrentDocument(CurrentDocument As LuceneNet.Documents.Document)
            Dim AddTime As String = LuceneNet.Documents.DateTools.DateToString(DateTime.Now, LuceneNet.Documents.DateTools.Resolution.MINUTE)
            CurrentDocument.Add(FieldFactory.Instance.CreateField(LuceneGeneralField.IndexedDate, AddTime))
            Dim currentModified As String = CurrentDocument.Get(LuceneGeneralField.ModifiedDate)
            If String.IsNullOrEmpty(currentModified) Then
                CurrentDocument.Add(FieldFactory.Instance.CreateField(LuceneGeneralField.ModifiedDate, AddTime))
            End If
        End Sub

        Private Sub CheckWhetherCommitIsNeeded()
            If Monitor.AddedDocuments + Monitor.UpdatedDocuments Mod CurrentPortalIndexingDriver.LucenePortalSettings.MaxNbDocPerCommit = 0 Then
                ' we batch write the added documents so far
                CurrentIndexWriter.Commit()
            End If
        End Sub

        Private Sub CheckStorageTimeAvailable()
            If Monitor.StorageTimeExceeded Then
                ' we exceeded storage time, warn users that we won't add anything anymore
                If CurrentPortalIndexingDriver.LucenePortalSettings.LogIndexingProcess Then
                    BusinessController.LogIndexingStep(LuceneLogType.Storage, CurrentPortalIndexingDriver.PortalId, _
                                            "TimeOut reached while indexing new documents. Remaining documents will be processed on next indexing session", _
                                            False, "Nb documents processed", Monitor.AddedDocuments)
                End If
            End If
        End Sub
#End Region

#Region "Logging"
        Private Sub LogMonitorInformation()
            If CurrentPortalIndexingDriver.LucenePortalSettings.LogIndexingProcess Then
                Dim MonitorLog As New Global.LuceneSearch.Monitoring.Logging.LogInfoProviderLogger(CurrentPortalIndexingDriver.PortalId)
                MonitorLog.LogInfo(Monitor)
            End If
        End Sub
#End Region

#Region "IDisposable Support"
        Private disposedValue As Boolean ' To detect redundant calls

        ' IDisposable
        Protected Overridable Sub Dispose(disposing As Boolean)
            If Not Me.disposedValue Then
                If disposing Then
                    'dispose managed state (managed objects).
                    CurrentIndexWriter.Optimize() 'optimize and close
                    CurrentIndexWriter.Dispose()
                    CurrentIndexWriter = Nothing
                    'shared reader are obsolete because structure changed, let them reload on next call
                    ' checking for updateddocuments is overkill since updating means deleting another document but we may have a point at which the distinction is necessary
                    If (Monitor.DeletedDocuments > 0 OrElse Monitor.AddedDocuments > 0 OrElse Monitor.UpdatedDocuments > 0) Then
                        CurrentPortalIndexingDriver.ReleaseState(False)
                        ' i'm forced to add this line because there is a discrespancy between indexingdriver and queryingdriver
                        ' ideally we shouldn't be needing this
                        'TODO: kill this call after reworking the wiring
                        PortalQueryingDriver.GetByPortalId(CurrentPortalIndexingDriver.PortalId).ReleaseState(False)
                    End If
                    CurrentPortalIndexingDriver = Nothing
                End If

                ' free unmanaged resources (unmanaged objects) and override Finalize() below.
                ' set large fields to null.
            End If
            Me.disposedValue = True
        End Sub

        ' override Finalize() only if Dispose(ByVal disposing As Boolean) above has code to free unmanaged resources.
        'Protected Overrides Sub Finalize()
        '    ' Do not change this code.  Put cleanup code in Dispose(ByVal disposing As Boolean) above.
        '    Dispose(False)
        '    MyBase.Finalize()
        'End Sub

        ' This code added by Visual Basic to correctly implement the disposable pattern.
        Public Sub Dispose() Implements IDisposable.Dispose
            ' Do not change this code.  Put cleanup code in Dispose(ByVal disposing As Boolean) above.
            Dispose(True)
            GC.SuppressFinalize(Me)
        End Sub
#End Region

    End Class

End Namespace